{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "# 网格搜索快速找到最优模型参数\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "from sklearn.linear_model.logistic import LogisticRegression\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = []\n",
    "x.append(\"fuck you\")\n",
    "x.append(\"fuck you all\")\n",
    "x.append(\"hello everyone\")\n",
    "x.append(\"fuck me\")\n",
    "x.append(\"hello boy\")\n",
    "x.append(\"fuck you\")\n",
    "x.append(\"fuck you all\")\n",
    "x.append(\"hello everyone\")\n",
    "x.append(\"fuck me\")\n",
    "x.append(\"hello boy\")\n",
    "x.append(\"fuck you\")\n",
    "x.append(\"fuck you all\")\n",
    "x.append(\"hello everyone\")\n",
    "x.append(\"fuck me\")\n",
    "x.append(\"hello boy\")\n",
    "x.append(\"fuck you\")\n",
    "x.append(\"fuck you all\")\n",
    "x.append(\"hello everyone\")\n",
    "x.append(\"fuck me\")\n",
    "x.append(\"hello boy\")\n",
    "x.append(\"fuck you\")\n",
    "x.append(\"fuck you all\")\n",
    "x.append(\"hello everyone\")\n",
    "x.append(\"fuck me\")\n",
    "x.append(\"hello boy\")\n",
    "y = [1,0,1,0,1,1,0,1,0,1,1,0,1,0,1,1,0,1,0,1,1,0,1,0,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipeline = Pipeline([('vect', TfidfVectorizer(stop_words='english')),\n",
    "                     ('clf', LogisticRegression())])\n",
    "parameters = {'vect__max_features': (3, 5)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   6 out of   6 | elapsed:    1.3s remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   6 out of   6 | elapsed:    1.3s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('vect', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), norm='l2', preprocessor=None, smooth_idf=True,\n",
       "  ...ty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))]),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'vect__max_features': (3, 5)}, pre_dispatch='2*n_jobs',\n",
       "       refit=True, return_train_score=True, scoring='accuracy', verbose=1)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search = GridSearchCV(pipeline, parameters, n_jobs = -1, verbose = 1, scoring = 'accuracy', cv = 3)\n",
    "grid_search.fit(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "最佳效果: 0.800\n",
      "最优参数组合: \n",
      "\tvect__max_features: 3\n"
     ]
    }
   ],
   "source": [
    "print('最佳效果: %0.3f' % grid_search.best_score_)\n",
    "print('最优参数组合: ')\n",
    "best_parameters = grid_search.best_estimator_.get_params()\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print('\\t%s: %r' % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 上面例子不够好，参考http://blog.csdn.net/jasonding1354/article/details/50562522再组织一下。\n",
    "# K折交叉验证回顾\n",
    "# - 交叉验证的过程\n",
    "#   - 选择K的值（一般是10），将数据集分成K等份。\n",
    "#   - 使用其中的K-1份数据作为训练数据，另外一份数据作为测试数据，进行模型的训练。\n",
    "#   - 使用一种度量测度来衡量模型的预测性能\n",
    "# - 交叉验证的优点\n",
    "#   - 交叉验证通过降低模型在一次数据分割中性能表现上的方差来保证模型性能的稳定性\n",
    "#   - 交叉验证可以用于选择调节参数、比较模型性能差别、选择特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "iris = load_iris()\n",
    "x = iris.data\n",
    "y = iris.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_neighbors': array([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,\n",
       "        18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_range = np.arange(1, 31)\n",
    "param_grid = dict(n_neighbors=k_range)\n",
    "param_grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise',\n",
       "       estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "           weights='uniform'),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'n_neighbors': array([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,\n",
       "       18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30])},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors=5) # param_grid自动匹配参数名字\n",
    "grid = GridSearchCV(knn, param_grid, cv=10, scoring='accuracy')\n",
    "grid.fit(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:747: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[mean: 0.96000, std: 0.05333, params: {'n_neighbors': 1},\n",
       " mean: 0.95333, std: 0.05207, params: {'n_neighbors': 2},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 3},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 4},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 5},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 6},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 7},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 8},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 9},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 10},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 11},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 12},\n",
       " mean: 0.98000, std: 0.03055, params: {'n_neighbors': 13},\n",
       " mean: 0.97333, std: 0.04422, params: {'n_neighbors': 14},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 15},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 16},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 17},\n",
       " mean: 0.98000, std: 0.03055, params: {'n_neighbors': 18},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 19},\n",
       " mean: 0.98000, std: 0.03055, params: {'n_neighbors': 20},\n",
       " mean: 0.96667, std: 0.03333, params: {'n_neighbors': 21},\n",
       " mean: 0.96667, std: 0.03333, params: {'n_neighbors': 22},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 23},\n",
       " mean: 0.96000, std: 0.04422, params: {'n_neighbors': 24},\n",
       " mean: 0.96667, std: 0.03333, params: {'n_neighbors': 25},\n",
       " mean: 0.96000, std: 0.04422, params: {'n_neighbors': 26},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 27},\n",
       " mean: 0.95333, std: 0.04269, params: {'n_neighbors': 28},\n",
       " mean: 0.95333, std: 0.04269, params: {'n_neighbors': 29},\n",
       " mean: 0.95333, std: 0.04269, params: {'n_neighbors': 30}]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.grid_scores_ # 查看所有分数，该属性已被cv_results_替换，可以提供更丰富信息。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_neighbors': 1}\n",
      "[ 1.          0.93333333  1.          0.93333333  0.86666667  1.\n",
      "  0.86666667  1.          1.          1.        ]\n",
      "0.96\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:747: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:747: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:747: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "# 查看第一组数据详情\n",
    "print(grid.grid_scores_[0].parameters)\n",
    "print(grid.grid_scores_[0].cv_validation_scores)\n",
    "print(grid.grid_scores_[0].mean_validation_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:747: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "grid_mean_scores = [result.mean_validation_score for result in grid.grid_scores_]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'Cross-Validated Accuracy')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xucm2d54P3fNUeNZ0aSD+OR7ZHj\nJOTk2J4xmJRzQlog9AAJAUqWtsCypdstvO0WWKD0ZWlKXlqgpe9uWbq0UMiWEiCcUhoaaEjoiUIc\nLNlOnBjHSayxZ+zxQZqTNTOSrv3jeTSWZUnz6DQaaa7v5zOfkZ6T7sca69J9um5RVYwxxphKtTW6\nAMYYY5qbBRJjjDFVsUBijDGmKhZIjDHGVMUCiTHGmKpYIDHGGFMVCyTGGGOqYoHEGGNMVSyQGGOM\nqUpHowuwHDZs2KDbtm1rdDGMMaapPProo6dVdWCp41ZFINm2bRt79+5tdDGMMaapiMizXo6zpi1j\njDFVsUBijDGmKhZIjDHGVMUCiTHGmKpYIDHGGFOVugYSEblFRJ4UkSMi8v4C+y8TkQdFZL+IPCwi\nQzn7PiYij4nIIRH5HyIi7vbnicgB95qL240xxjRG3QKJiLQDnwJeDWwH7hCR7XmHfQK4W1V3AXcC\nH3XPfRHwYmAXsAN4PnCje86ngXcAV7k/t9TrHowxxiytnjWSG4AjqnpUVeeBe4DX5h2zHXjQffxQ\nzn4FfEAX0A10AidFZBPgV9UfqrNG8N3ArXW8B7MKPfrsOSKxeKOLUTNzqTRf+vEx0pnaLaudXEhz\nz4+PkanhNU3zqmcg2QLEcp6PuttyRYHb3ce3Af0isl5Vf4gTWMbcnwdU9ZB7/ugS1wRARN4hIntF\nZO/ExETVN2NWj/d9bT8f/MaBRhejZu4/MMYHvn6Af/5p7f4ffHv/GO//+gH2xc7V7JqmedUzkBTq\nu8j/+vIe4EYR2YfTdHUcSInIc4DrgCGcQHGziLzM4zWdjaqfUdU9qrpnYGDJGf7GADCZXOCpiWme\nGJ/i/Hy60cWpicgxp3ZVy1pWxA0gJ+LJml3TNK96BpJRIJzzfAg4kXuAqp5Q1dep6m7gg+62BE7t\n5N9VdVpVp4HvAC9wrzlU6prGVOPAaAJVSGeUx04kGl2cmoiMOvcRrWEgicaca56ctEBi6htIHgGu\nEpHLRaQLeBNwX+4BIrJBRLJl+ADwOffxMZyaSoeIdOLUVg6p6hgwJSIvcEdr/RrwrTreg1llcr+1\nt0I/yVwqzaETkwBERxM4XYvVSS6kOTTmXHMsYYHE1DGQqGoKeCfwAHAI+IqqPiYid4rIa9zDbgKe\nFJHDwCBwl7v9XuAp4ABOP0pUVf/O3febwF8BR9xjvlOvezCrTyQW5/INvWwJ9rREIDk0NsV8OsNL\nr9rA2Zl5YmfPV33Nx05MknI72cetRmKoc/ZfVb0fuD9v24dyHt+LEzTyz0sDv1HkmntxhgQbU1Oq\nSiQW5yXP2cB8KkN0tPkDSbY56y0v3MY///Q0kdE4W9evqck1r9jQy0mrkRhsZrsxi8Ynk0xMzTE8\nFGA4HCB29jxnpucaXayqRGNxBvq7ufGaAXydbTXpJ4mOxgn5fewaCliNxAAWSIxZlB3dNBwOMjwU\nBGj6WkkkFmd4KEhnexs7Ngdq0lwXicUZDgcIBXo4NTlnc0mMBRJjsiKjcTrbhe2b/ewcCtAmEIk1\n78itxOwCR0/PsHurExRHwkEOHk+wkM5UfM1zM/M8e2aWkfBaQv5u5tMZzs7O16rIpklZIDHGFTkW\nZ/smP90d7azp6uDqwf6m7nDP1qaytavhcJC5VIYnx6cqvmYke81wgFDAB8C49ZOsehZIjMGZN3Lg\neIKRcHBx2+6tQaKxeE2GzDZCtj9kVzgAsHhv1QTHaCyOCOzcEmDQ7wQSm0tiLJAYAxw5Nc3sfJrh\nnEAyPBQkcX6BZ8/MNrBklYuOxrlyoBe/rxOAobU9rO/tqqrDPRqLc9XGPvp9nRdqJBZIVj0LJMZw\nIeXHRYGkBt/gGyU7lDn3fkSE4XCw4vtZvKbbVDbQ102bYEOAjQUSY8DpVPf7Orh8fe/itqsH+1nT\n1d6UgeR4/Dynp+fZnRNIwGneOjIxzVRyoexrxs6e59zswmJw6mhvY6C/22okxgKJMeA02QyHg7S1\nXcgL2t4m7NhSmyGzyy1b5uG8QDIcDqIKB46XPxot29Ge248U8vssTYqxQGLM+fk0T56cWmyyyTUS\nDvL4iUnmU5UPmW2EaCxOV0cb14b8F20fHnI63isJjpFjcbo72rgm1L+4bdDvs852Y4HEmIMnEqQz\netE37ayRcJD5dIYnxicbULLKRWMJrt/sp6vj4v/iwTVdXL6ht6IO9+honJ1bAnS2X7hmKOCz4b/G\nAokx0SLNQLnbmql5K5XOcOB4omANC5xaSbTMiZYL6QwHjycu+Tca9PuYTKZaZu0WUxkLJGbV2xeL\nsyXYw0B/9yX7Ngd8DPR3N1UgOXxymvML6cUZ7flGwkHGJ5Nl1SSeHJ9iLpW5JJBssiHABgskxhCN\nxQs2a4E7ZHYoWNNFoeotf0Z7vkpqWdlj80eBhdxJiWOJ6tPTm+ZlgcSsaqen5xg9d55hd/Z3ISPh\nAE9NzJA4X/6Q2UaIHIsTXNPJZUXSxV+3yU9nu5QdSNb1djG0tuei7YMBm91uLJCYVS5b0xgJry16\nTHbfgdHmSOAYHXUmDTqLiF7K19nO9k3+smpZ0Vic4aHAJdfM1kjGE82dbt9UxwKJWdWisThtAju2\n+Ises3NxyOy55SpWxWbmUhw+OVVw4ECu4XCQA8ed0WpLmUoucGRiumCw7e3uoL+7w2okq5wFErOq\n7YvF3RnsxRcLDfR0csVAb1OklD9wPEFGnea4UoaHgkzPpXhqYnrpa44mUKVo89+gDQFe9SyQmFVL\nVYnG4kVHN+UacXNUrfRMwItDmYt0tGeNbPXe4V5oRnuuTQEfY1YjWdUskJhV65kzs0wmU0t+6ILz\nIXp6eo4TK/ybdyQWJ7yuh/V9lw5lznX5+l76fR2e+kmisTjb1q8huKar4P5Bv88SN65yFkjMqpXt\n8xjxWCMBVvwwYGcoc/GBA1ltbbJYy1pKfhbhfCG/j4npOU/9LaY1WSAxq1Y0lmBNVztXbexf8thr\nQ3662ttWdCA5NZnkRCK5mE9rKcNDQZ4YnyK5UHxW+ngiycnJuaLNWuD0kaQzyulpG7m1WlkgMavW\nvlicHVsCtLcVHiabq6ujje2b/exbwYEkEivdl5FvOBwknVEeO1F8EEGxLMK5LgwBtuat1coCiVmV\n5lJpDp2YvGSmdikj4SAHRhOk0iszE3B0NL6Y+t6L7CisfceKB8dILE5nu7B9U/Hh0dk0KZZOfvWy\nQGJWpSfGpphPX5o7qpSRcJDzC2l+emrpIbONEInFuTbUj6+z3dPxG/t9bAn2EC0x0TIai3PdJn/J\na9ra7cYCiVmVvDTZ5BtewR3umYyyP3Zpdt6lDIcDRSdapjPK/tH4kqPa1vd20dkulrhxFbNAYlal\naCzOQH83m91mGS+2rV9DoKdzMSniSnL09AxTcynP/SNZI+EgsbPnOVOgo/ypiWlm5tNLXrOtTdjY\nb0OAVzMLJGZViiyRj6oQEWE4HCzZp9Ao5Xa0Z2VrG/sLNG+VU2sb9Nva7atZXQOJiNwiIk+KyBER\neX+B/ZeJyIMisl9EHhaRIXf7y0UkkvOTFJFb3X2fF5Gnc/aN1PMeTOtJzC5wdGLG04z2fCPhIIdP\nTjE7n6pDySoXjcXp6+7gyoG+ss7bORSgTSg4Gi0Si9Pv6+CKDb1LXicU8FkgWcXqFkhEpB34FPBq\nYDtwh4hszzvsE8DdqroLuBP4KICqPqSqI6o6AtwMzALfzTnvvdn9qhqp1z2Y1rT/uLc0IoWMhANk\nFA4eX1lL72aXwfUylDnXmq4Orh7sL9jv42T8DdLm4Zohfw/jieSKTyFj6mPJQCIivygilQScG4Aj\nqnpUVeeBe4DX5h2zHXjQffxQgf0Arwe+o6qzFZTBmEtkPzR3epy4lysbfFZSJuDkQppDY5Nld7Rn\njYSDREcvziOWXEjzxPhUyXVacoUC3czOp5maW1k1NbM8vASINwE/FZGPich1ZVx7CxDLeT7qbssV\nBW53H98G9IvI+gKv/6W8bXe5zWGfFJGCSYVE5B0isldE9k5MTJRRbNPqIrE4Vw70EujpLPvc9X3d\nhNf1lL3meT09PjbJQlrL7h/JGgkHic8u8OyZC9/VDrop5r2kW4GcIcDW4b4qLRlIVPVXgN3AU8Bf\ni8gP3Q/ppfJKFKoP59d73wPcKCL7gBuB48DiVxoR2QTsBB7IOecDwLXA84F1wPuKlPszqrpHVfcM\nDAwsUVSzWqgqkQqGyeYaHvKWo2q5RI5V1tGetTisOWc02mJHu8da2+LsdusnWZU8NVmp6iTwNZzm\nqU04tYefiMi7Spw2CoRzng8BJ/Kue0JVX6equ4EPuttyv+q9EfiGqi7knDOmjjngr3Ga0Izx5Hj8\nPKenS+eOWspIOMjx+HlOTa2MD83oaJxBfzehMoYy57pqYx89ne0XjUaLxOJsDvjY6Pd2zexrW5qU\n1clLH8kvicg3gO8DncANqvpqYBinRlHMI8BVInK5iHThNFHdl3ftDTn9Lx8APpd3jTvIa9ZyaymI\nM27zVuDgUvdgTFa2SaraQAKwf4U0bzkZfyu/n472NnYOBS6qkURH456yImcNWr6tVc1LjeQNwCdV\ndZeqflxVTwG4nd//sdhJqpoC3onTLHUI+IqqPiYid4rIa9zDbgKeFJHDwCBwV/Z8EdmGU6P5Qd6l\nvygiB4ADwAbgIx7uwRjA+YDs6mjj2lDx3FFLuX6zMzpqJTRvnZuZ55kzs1U11YETHB87Mcl8KsOZ\n6TliZ8+XNarN19nO2jWd1rS1ShVfX/SC/w6MZZ+ISA8wqKrPqOqDxU8DVb0fuD9v24dyHt8L3Fvk\n3Ge4tHMeVb3ZQ5mNKShyLM71m/10dVQ+8r2nq51rQ/0rYoZ7dInVC70aCQeZT2V4YnxyMR18ucFp\n0O+zfFurlJf/TV8FctOdpt1txjSVVDrDgeOJiuaP5BsOB4nG4mQavJhTNJZABHZ6zPhbTG4esUgs\nQVsF17RJiauXl0DS4c4DAcB9XHjNTWNWsJ+emub8wtK5o7wYGQoymUzx9JmZGpSscpHYOZ4z0Ee/\nr/yhzLk2B3xs6OsmEksQjcW5erCf3m4vDRYXhPw+xhO2uNVq5CWQTOT0aSAirwVO169IxtRHpfmo\nCsl2RDcyE7CqEh1N1OR+RJyld/fFzjkd7RVcc9Dv4/T0HPOplblei6kfL4HkPwO/JyLHRCSGM2/j\nN+pbLGNqLxqLE+jp5LL1a6q+1pUDffR2tTe0wz129jxnZ+ar7mjPGgkHODoxQ3x2oaJrZhe4WinD\nos3yWbLuqqpPAS8QkT5AVHWq/sUypvYisTjD4fIy/hbT3ibOkNkGBpJIjTras3KDRyX9SIOBCwtc\nDa2tPlib5uGpEVREfgG4HvBl/xOq6p11LJdpQaqKKp6SANbazFyKwyeneOX1oZpdcyS8ls/+y1HG\nEufpaFv+FRl+/PQZujvauCa0VJIJb3a5waOns52rB8vLIgy5a7fXv58kk1FEqMmXAlO9JQOJiPwF\nsAZ4OfBXOEkUf1zncpkW9K4v7UMVPvXm5y77ax88niCj3lN+eLF7a5CFtPLCj36/Ztcs157L1tLZ\nXpsgFujp5Dkb+1jf20VHBddczjQpb/38I1y2bg1/eOuOur+WWZqXGsmLVHWXiOxX1T8QkT8Bvl7v\ngpnWkskoPzg8Aeo8Xu5aSS072rNuvnYjn3jDMOcX0jW7ZrlecPm6ml7vU//huXRXOMcmuKaTro62\nus8lSS6k+eFTp5k8X7svBaY6XgJJ9q9iVkQ2A2eAy+tXJNOKnj4zw1TSycd59PQ0z9lYm+YYr6Kj\nccLreljfVzBZdEU629t4/fOGana9laCaZjIRYVPAx1id06Rksx3b5MeVw8tXj78TkSDwceAnwDNc\nmtbdmJJyO6UjDchRFY3VZiKiKW3QX/+127N/S6em5kg3eEKocZQMJG5CxQdVNa6qXwMuA67NTXNi\njBeRWJzernb6ujuWfVGoU1NJjsfP17RZyxQW8td/dnu2mTKdUc5M2wTIlaBkIFHVDPAnOc/n8tK8\nG+NJNBZn11CQXUOBZV8UqhYZf4032TQp9VxyNxqLs6arHaDuzWjGGy9NW98VkdvFxtmZCiUX0jzu\nLgU7HA5yaGyS5DJ2UEdjcdrbhOs3W+dsvQ36fcynMsRnF5Y+uALZbMcvv2YjYAtprRReAsnv4iRp\nnBORSRGZEpHJOpfLtJBDOUvBjoSDpDLKYyeW708oEotzbaifHvdbrKmf7BDgetUUstmOX7XDmQ9k\nHe4rg5eldvtVtU1Vu1TV7z6vfDEHs+pEc4bejoSXN0dVJqNER+M1SyNiSgvlzG6vh2y245uuGaCj\nTWwhrRXCy4TElxXarqr/VPvimFYUiV28FGzI71u2HFVHTzvDjkdsxNayWFxyt06BJBI7x1Ub+/D7\nOhlcho59442XeSTvzXnsw1kj/VHAFpgynuRnqB0JB5dtUajF2lAZy8aaym3s70akPkvuZrMd/9x1\nTv/IoL/baiQrhJemrV/K+XkFsAM4Wf+imVYQn53n6dMzFycEDAd59sws52bmS5xZG9FRZ9jxlQPl\n544y5etsb2N9b3ddmrZGz12c7dgW0lo5KsmFMIoTTIxZUnT00qG32ceRZaiVRNxhx+0NSBS5WoUC\n3XXpbN/n1i6zE0uXY/Kj8cZLH8n/BLKDwtuAESBaz0KZ1hGNxS9ZCnbnUAARZ192GGc9JBfSHBqb\n5O0vuaJur2EuFfL3MHputubXjcbi+DovZDsO+X3MzKeZSi5UvUKkqY6XPpK9OY9TwJdU9V/rVB7T\nYiKx+CVLwfZ1d3DVxr66d7hfGHZs80eWUyjQzd5nz9b8upFYnB2bA4vZjnNHiFkgaSwvgeReIKmq\naQARaReRNapa+68cpqWoKtFYnJuvvbTWMRIO8r3HT6KqdVtT4kLG37V1ub4pLOT3EZ9dILmQxtdZ\nm7k7C+kMB48n+JUXXHbR64AzZ2W5k4Cai3npI3kQ6Ml53gP8Y32KY1rJ6LnznCmyFOxwOMi52QVi\nZ8/X7fWjecOOzfIY9Nd+LsmT41PMpTIX9bUtDjW2fpKG8xJIfKo6nX3iPrZ1NM2SSq0Bku0w3VfH\nBI7RUcv42wjZD/hadrgX+luqR8AylfESSGZEZHFJOxF5HlC/r5GmZURj8aJLwV4T6sfX2Va3BI7Z\nYcc2f2T5herwAR+NxVnX28XQ2guNI77OdoJrOm0I8ArgpY/kd4CvisgJ9/km4JfrVyTTKiKxODu2\nBAouBdvZ3saOzYG6pZRfHHZsNZJlV48mp0gszkg4eEl/WsjvW5Y14k1pXiYkPgJcC/wm8F+A61T1\n0XoXzDS3hXSGgycSJVO3j4SDHDwxyUI6U/PXjxxzhx3XcI12402/r5Pervaa1RSmkgscmZgu2Ezp\nTEq0BpJGWzKQiMhvAb2qelBVDwB9IvJf6l8008wOn5wiuZApmSxxOBxkPpXhyfGpmr9+dPTSYcdm\n+QwGfDVr2jpwPIEqDBcYxm01kpXBSx/Jr6vq4oB/VT0H/LqXi4vILSLypIgcEZH3F9h/mYg8KCL7\nReRhERlyt79cRCI5P0kRudXdd7mI/EhEfioiXxaRLm+3apbTYudoiaalbG1lX43nk2SHHVvG38Zx\nPuBrE0hKDdoY9Ps4MzNXl1qt8c5LIGnLXdRKRNqBJT+83eM+Bbwa2A7cISLb8w77BHC3qu4C7gQ+\nCqCqD6nqiKqO4CSHnAW+657zx8AnVfUq4Bzwdg/3YJZZtnM0vK6n6DFDa3tY39tV85Ty2WHHtiJi\n49QykERjcbatX0NwzaUfO6GAD1Vn/XbTOF4CyQPAV0TkZ0XkZuBLwD94OO8G4IiqHlXVeeAe4LV5\nx2zHmacC8FCB/QCvB76jqrNuQLsZZ5IkwBeAWz2UxSyzaCzB8FCg5GRDEWE4HKx5ICn1DdYsj1DA\nx6mpOTKZ6pfcjcYSRWuX2RFiNpeksbwEkvfhfNj/JvBb7uP3ljzDsQWI5TwfdbfligK3u49vA/pF\nZH3eMW/CCV4A64G4qqZKXBMAEXmHiOwVkb0TExMeimtqZXouxeFTU56aloaHghyZmGYqWbulWUsN\nOzbLIxTwkcoop2eqqymMJ5KMTyaLfikYtECyIngZtZVR1b9Q1der6u3A/cC7PVy70FfR/K8n7wFu\nFJF9wI3AcZx8Xs4FRDYBO3FqRV6vmS33Z1R1j6ruGRgY8FBcUysHRp3OUS81gpGtQVSdc2ql1LBj\nszwWJwtW2RGerV0W+1Kyqc4LaRlvPP1PE5ENIvKbIvJPwMPAoIfTRoFwzvMh4ETuAap6QlVfp6q7\ngQ+623I/Ud4IfENVs19XTwNBEcnOf7nkmqbxInnpvksZdofn1qrDPTvs2Ga0N9Zik1OVH/CRWJzO\ndmH7psKrewfXdNLV0Waz2xusaCARkX4R+TUR+Qfgx8BzgCtU9UpVfY+Haz8CXOWOsurCaaK6L+81\nNohItgwfAD6Xd407uNCshaoqTl/K691NbwG+5aEsZhllO0fX9i49oC64povLN/TWrJ8kO+zYZrQ3\n1oVJidXN8YjG4ly3yV80+aOI1LRj31SmVI3kFM6IqLuAK1X13YDnJe3cfox34jRLHQK+oqqPicid\nIvIa97CbgCdF5DBOLeeu7Pkisg2nRvODvEu/D/hdETmC02fyWa9lMssjOlre0NvhoUDNlt71MuzY\n1N+Gvm7a26SqGkk6oxw4vnTtMmRrtzdcqRQpv4dTi/g08Lci8uVyL66q9+P0qeRu+1DO43u5MAIr\n/9xnKNCRrqpHcUaEmRXo5GSSsUSyrKal4XCQb0ZOMJY4z6ZA8eHCXngZdmzqr71N2NjfXdVkwacm\nppmeSy3Z1zYY8LF/GVbbNMUVrZGo6idV9WeA1+B0cn8T2Cwi7xORq5ergKa5LNYIymhayn5Q1KJ5\ny8uwY7M8Bv3VzW5fqqM9a1PAx1giidPybRrBy6ito6p6l6ruBJ4PBIDv1L1kpilFYnE62op3jhZy\n3SY/ne1SdYd7OcOOTf1V2+QUicXp93VwxYbekscN+n3MpzLEZ2s3hNyUp6zxkap6QFV/T1WvrFeB\nTHNbqnO0EF9nO9dt8lddI8kOO7ZAsjKEAtV1gkdjcYaHgrS1la5d1mqEmKmcDbQ3NZPJKPtHS2f8\nLWYkHOTAaIJ0FTOhraN9ZRn0+5ieSzE9l1r64DzJhTRPjE8VTNSYLxToBiyQNJIFElMz2c7RSmoE\nw0NBZubTHDk1vfTBRURjcS7zOOzY1N+mKtYlOXjc+VLhZdDGhcmPFkgaxQKJqZlqclxlO+erad6K\njsYtv9YKUs1SuOX8LW3sr/3SvqY8RYf/isgBiqQfAXAz9hqzKDoap7976c7RQi5f30u/r4PIaJw3\nPj+89Al5Khl2bOqrmpUSo6MJNgd8bHSDUSldHW1s6Ou22e0NVGoeyS+6v3/L/f1/3N9vxknrbsxF\nIrE4u8KBJTtHC2lrE4aHgkSOVVYj8TpU1CyfajrBI7FzZb2XoUC39ZE0UKl5JM+q6rPAi1X1v7kj\ntg6o6vuBVy1fEU0zSC6keWJsqqqmpZFwkCdPTnF+Pl32uVF32PH1m70POzb11dPVjt/XUXaN5Mz0\nHLGz58v6W7I0KY3lpY+kV0Rekn0iIi8Cym+7MC3tsRMJUh47R4sZDgdJZ5SDJ8rPBBypYNixqb9N\ngZ6yawrZdDnl1EiqnfxoquMlkLwd+JSIPCMiTwP/C/iP9S2WaTaRmPPhX02NJDvUs9wO9+ywYy9D\nRc3yqmTt9kgsQZvAzi3e38+Q38e52QWSC+XXZk31SvWRAKCqjwLDIuIHJC/NuzGA8+HvtXO0mI39\nPrYEexb7O7w6ejqbk2ltxa9t6iPk7+aJscmyzonG4lw92E9v95IfTxdeJ3BhhNhl663BZLktWSMR\nkUER+SzwZVVNiMh2EbF10s1FIrHyMv4WMxwOlB1I9h3LDhW1GslKE/L7mJieYyGd8XS8qjrZo8ts\nIq1mhJipnpemrc/jpILf7D4/DPxOvQpkms/ZmXmOnZ2tyRyOkXCQ0XPnOT3tPWvshWHHfVW/vqmt\nwYAPVZiY8vZ+PntmlvjsQtnryVialMbyEkg2qOpXgAwsrjNiDZFmUbSGQ2+z30TL6SepZtixqa9y\nP+DLWV0z12Cg8smPpnpeAsmMiKzHnZwoIi8ArJ/ELIrE4mV3jhazY0uANvEeSLLDjm0i4sq02Hfh\nsckpEovT09nO1YPl1S77uztY09Vus9sbxEtv1u/iLJF7pYj8KzAAvKGupTJNJTpafudoMb3dHVw9\n2E9k1Nt3lcdOTJLKqKVGWaHKrZFER+Ps3BKgo7287E3ZJXetRtIYXt6tx4AbgRcBvwFcDzxRz0KZ\n5qGqi+m+a2UkHCQai3taqKia/F6m/tb1dtHV3uYpkMynMjx2YrLiYdzVpq03lfPyFfKHqvpcnIAC\ngIj8BHhu3UrVwr5zYIz7oicaXYyaWUhnODe7UNPUJMPhIPc8EuM/fWEvXR2lv+s8dmKSTVUOOzb1\nIyJs9Hfz7egYx86Uzqw0O59mPpWp+G8p5Pfxo6fPVnQuwFf3xljf18XN1w5WfI1C1xzo7+amazbW\n7JorUamkjSGcNdN7RGQ3znK7AH5gzTKUrSV99l+e5tDYJFvWts6a4sNDAW66ZqBm17vpmgGGw0Fi\n55ZO6ebrbOOXK0jyaJbPbbu38MBj4zw1sfQSAc/dGuTFV26o6HWykx8zGS174IWq8pG/P8S2Db01\nCySZjPKH336c4XBw9QYSnHxabwWGgD/N2T4F/F4dy9TSxhJJXnl9iE/+8kiji7JibQr08K3fenGj\ni2Fq5N2vvIZ3v/Kaur9OyO8jlVFOz8wtppb36tkzsyTOL3DoxCRzqTTdHdWn2nnmzAyTydSq6Lcp\nGkhU9QvAF0TkdlX92jKWqWVPWac8AAAfnklEQVRlMsqpqeTiSBZjTO1cWOCq/ECS7WubT2c4VGXy\n0fxrroZ+Gy8pUr4mIr+A08nuy9l+Zz0L1orOzs6zkNbFkSzGmNpZXJFxMslOyuuwj7jZo1MZZ/BI\nLQJJdgj7ZDLF+fk0PV2tm1DUS4qUvwB+GXgXTj/JG4DL6lyulpT9ZjJogcSYmgsFKp/dHh2N89yt\naxno765qlc5cuUPYW33GvZfhvy9S1V8DzqnqHwAvBKx3swLZtlJr2jKm9jb0ddPeJmWv3Z4ddjyy\nNchIOFh2rrdC5lJpDp2YXJyk2+rNW14CyXn396yIbAYWgMvrV6TWlZ11a01bxtRee5sw0Ndd9uz2\nJ8YnnWHHQ04gOXp6hsTsQlVlOTQ2xXw6wy07QgCMT55f4ozm5iWQfFtEgsDHgZ8AzwD31LNQrerk\nZJI2gQ19XY0uijEtqZL1Ty7kigtcyPU2Wl2tJHLsHACvut4ZSjye8J6EtBl56Wz/Q/fh10Tk24DP\n1iSpzHgiycZ+X9npH4wx3oT83Tw1MVPWOfticTb0dbMl2IO/pxNwgsvLrq58blR0NMHG/m6uHOij\nr7uj5YcAl5qQ+LoS+1DVry91cRG5Bfj/gXbgr1T1j/L2XwZ8Did/11ngV1R11N23FfgrnP4YBX5e\nVZ8Rkc/jpGzJBrO3qmpkqbKsBOOTycUspcaY2tsU6OHfjpwp6xxnlFYAEcHv6+TKgd6qayTZkV8i\nwqC/u+X7SErVSH7J/b0RJ8/W993nLwceBkoGEhFpBz4FvAIYBR4RkftU9fGcwz4B3K2qXxCRm4GP\nAr/q7rsbuEtVvycifbhp7F3vVdV7l7q5lebkZJLLN9jqbcbUy6Dfx9Rcipm5lKckopPJBZ6amOG2\n3VsWt42E1/KDw6dQVUTKX5ogMbvA0dMz3P68IaCydeubTdE2FlV9m6q+Dac2sF1Vb1fV23Hmk3hx\nA3BEVY+q6jxOv8pr847ZDjzoPn4ou19EtgMdqvo9tyzTqrp0vowVbiyRtI52Y+ooFOgGvA+33R9z\nGjZy83uNhAOcnp7neLyyDvJsbSY7F2VwFWQl9tJYv01Vx3KenwSu9nDeFiCW83zU3ZYrCtzuPr4N\n6HfXPrkaiIvI10Vkn4h83K3hZN0lIvtF5JMi0l3oxUXkHSKyV0T2TkxMeChufc3Op5hKpqxpy5g6\nys7R8tqUlP3Q35WTvTobVKKxyrqCo7E4IrBzyBn6Gwp0c2pqjnRm6WzWzcpLIHlYRB4QkbeKyFuA\nv8epPSylUJ0w/1/yPcCNIrIPp9/jOJDCaXJ7qbv/+cAVOHm/AD4AXOtuXwe8r9CLq+pnVHWPqu4Z\nGKhdQsFKjdvQX2PqLlRmINl3LM4VA70E3E52gGtDfro62ojEzlVUhkgszpUDffh9nYtlSme0rOWj\nm82SgURV3wn8b2AYGAE+o6rv8nDtUS6euDgEXJQ/XVVPqOrrVHU38EF3W8I9d5/bLJYCvombtl5V\nx9QxB/w1ThPaijdukxGNqbtyZrerKpFYnJG8tXS6Otq4frO/ohqJqhIdvTjFSrm1pGbkaRyqqn5d\nVf+r+/MNj9d+BLhKRC4XkS7gTTgrLS4SkQ0iki3DB3BGcGXPXSsi2arEzcDj7jmb3N8C3Aoc9Fie\nhlqc1W41EmPqZk1XB36ft+G2Y4kkp6fnCq5/MjwU5MDxBKl0psCZxR2Pn+f09PxF19wUcJaMaOUO\n96KBRET+xf09JSKTOT9TIjK51IXdmsQ7gQeAQ8BXVPUxEblTRF7jHnYT8KSIHAYGgbvcc9M4zVoP\nisgBnGayv3TP+aK77QCwAfhI2XfdANkJSVYjMaa+vK6UWGp1zd1bg5xfSHP45NJrqBS8Zk4tZ9Ad\nANDKHe6l0si/xP3dX+nFVfV+4P68bR/KeXwvUHAYrztia1eB7TdXWp5GGk+cp9/XwZqu6tc1N8YU\nN+j3efr2H43F6Wpv49pNl37E5c5w377Z7/m1o7E4XR0XX3NDbzcdbVJ26pZmUqpGsq7Uz3IWshWM\nT9rQX2OWQ8jvvUZy3WZ/wUWsLlu/huCaTiLHypuYGInF2bHZT2dO9oq2NmFjf3fZySSbSamvx4/i\njLIqNvrqirqUqEWNT85Zs5YxyyAU8HF6eo5UOlM0HVE6oxw4nuCNewonMhcRhoeCZc1wT6UzHDie\n4I4btl6ybzDgrZbUrEpNSLxcVa9wf+f/WBAp00mbjGjMsggFfGQUJkoMt/3pqSlm59MMh4svgDUc\nDnL45BQzcylPr3v45DTJhUzBPpdNqzWQ5BKRtSJyg4i8LPtT74K1klQ6w8S01UiMWQ5e5pJEFzva\n1xY9Znc4SEbhwHFvw4BLdd4P+n0t3bTlZYXE/wT8E87oqz9wf3+4vsVqLaen50ln1FZGNGYZeJm3\nEYnF8fs62LZ+TdFjdrkz072umBiNxVm7ppOt6y69ZsjvY2Y+zVSyunVOViovNZLfxplF/qyqvhzY\nDTQ+50gTGbc5JMYsGy+TEiOxBMNudt5i1vd1E17X47mfJDoaL3rNxTK1aK3ESyBJqmoSQES6VfUJ\n4Jr6Fqu1LKZHsaYtY+pu3ZouOtulaCCZnU9x+OQUuws0QeUbCa/1NHJrZs655vBQ4Wsu1pJatJ/E\nSyAZdVdI/CbwPRH5FnmpTkxptla7McunrU1K9kkcPD5JOqMFZ7TnGx4KcCKR5NQSAeDA8QQZLdw/\nAuXnAGs2XlZIvM19+GEReQgIAP9Q11K1mPHJJJ3twro1tsSuMcshVGJS4oWldb3USJxjIrE4r7w+\nVPS4yBLXzH6JbNXZ7aUmJP69iLxZRBZXYlLVH6jqfe76Isajk+4Su21t5S+SY4wp32CJNCmRWJyh\ntT1s6Cu4AsVFdmwJ0N4mS/aTRGNxtq5bw7rewl8WfZ3tBNd0rsqmrc8Avwg8IyJfFpFb3eSLpkxj\niaQ1axmzjLI1EtVL1wCJxOKeaiPgBIBrQ/1LZgLOLq27ZJkSrZlKvtSExG+p6h3AVpxldd8CHBOR\nz4nIK5argK3gpKVHMWZZhfw+kgsZJs9fPJlwYmqO4/Hznjras0bCQaKxOJkiC1OdmkxyIpFcMjg5\nOcAqW3VxpfOyHsl5Vf2y21fySpzhv9ZH4pGqOnm2rEZizLIZLDIEuJz+kazhcJCpuRRHT88U3H9h\nImLxWfKwSmskWSIyKCLvEpF/xRm59V3geXUvWYuYmksxO5+2Gokxy2hTsUAyGqe9TdixufSHfq6R\nxaV3C/eTREfjdLQJ1y9xzVDAx5mZORbKXOOkGZTqbP91Efk+8BOcNdT/m5t7632qGlm2Eja57BBE\nW6vdmOVzYbjtxU1JkVicawb76em6NONvMVcO9NHX3bFY88gXicW5dlM/vs7S1wwFfKjCqanWq5WU\nqpG8CPgjIKyq71LVf12mMrWUMVur3Zhlt9HvjMjKbUrKZJRoGR3tWe1tws4tgYIjtzIZZX8sUXQi\nYq5WnktSqrP9bar6XVVdrIeJyIeXpVQtxNKjGLP8ujvaWdfbdVHT1jNnZphMppbsyyhkOBzk0Ngk\nyYX0RduPnp5mai615IgtaO212z1l/83xmqUPMbmyTVvZb0jGmOUx6PddNAEwW6MolfG3mJFwkIW0\n8vjYxauMR9xhwV4CiZccYM2q3EBiM+rKND6ZZF1v15Ltp8aY2tqUNykxcixOb1c7z9nYV/a1inW4\nR2Nx+ro7uGJg6WuuXdNJV0dbS85uLzeQ2GitMp2cTFr6eGMaIL9GEhlNsHPImalerlDAR8jvu6TD\nPRKLs8vjNUXE8zLAzcbL8N+PiYhfRDpxkjaeFpFfWYaytQRnrXZr1jJmuYX8Ps7MzDOXSjOXSnPo\nxGTZHe25hsOBi2okyYU0h8bKu2apHGDNzEuN5JWqOomTLmUUZyjwe+taqhYybulRjGmIUMD5Andq\nco5DY1PMpzOMeBhdVcxwOMgzZ2aJzzqpBh8fmySVUU8jtrIGA75V27TV6f7+eeBLqnq2juVpKfOp\nDKen561py5gGyF0DZHFp3a2VB5LcTMDA4jolu8u4ZsjfzViicA6wZuYlkPydiDwB7AEeFJEBoPVC\nah2cmrKhv8Y0yqZAD+C0CkRicTb2d1f1f3HnlgAiLCZwjI7GCfl9ZX1RDAV6mE9liM+21pK7XnJt\nvR94IbBHVReAGeC19S5YK7AFrYxpnGzQOOnWSJZaWncp/b5OnjPQRyR2DnBqJl6G/RYqU6v1k3jp\nbH8DkFLVtIj8PvA3wOa6l6wFZGfVWiAxZvn5ezrwdbbx5PgUR0/PlP2hX8hIOEh0NMHZmXmePTNb\ndud9tt9m1QUS4P9V1SkReQnwKuALwKfrW6zWMObm+bGmLWOWX3a47YNPnAK8TRpcynA4yNmZef7+\nwJj7vLxZ8tlmsGLLADcrL4EkmxPgF4BPq+q3AFvgyoOTk0m6O9oI9HQufbAxpuYG/T7OzswjAjuH\nyk+Nki8bjO7+t2cQgV1ljgLb2O8EkrFVGEiOi8j/Bt4I3C8i3R7PQ0RuEZEnReSIiLy/wP7LRORB\nEdkvIg+LyFDOvq0i8l0ROSQij4vINnf75SLyIxH5qbty44oNauOTc4QCvqraZY0xlcs2K1850Iff\nV/0XumtC/XR3tPHTU9NctdHJClyOro42NvR1t9wQYC8B4Y3AA8AtqhoH1uFhHomItAOfAl4NbAfu\nEJHteYd9ArhbVXcBdwIfzdl3N/BxVb0OuAE45W7/Y+CTqnoVcA54u4d7aIiTCVsZ0ZhGygaScuZ6\nlNLZ3saOLYGqrhkKdK++PhJVnQWeAl4lIu8ENqrqdz1c+wbgiKoeVdV54B4uHe21HXjQffxQdr8b\ncDpU9XtuGaZVdVacr/Y3A/e653wBuNVDWSry9OkZfnLsXMXn28qIxjRW9otcNfNH8mWbtyq9Zium\nSfEyauu3gS8CG92fvxGRd3m49hYglvN81N2WKwrc7j6+DegXkfU4s+fjIvJ1EdknIh93azjrgbiq\npkpcM1vud4jIXhHZOzEx4aG4l/rQtw7y+984WNG5i0vsWo3EmIa5cqAPEbhh27qaXfNnLl+HCDy/\nwmvm5wBrBV6att4O/IyqfkhVPwS8APh1D+cV6hjIn875HuBGEdkH3AgcB1JAB/BSd//zgSuAt3q8\nprNR9TOqukdV9wwMDHgo7qWGh4I8eXKK8/PppQ/Oc252gflUxma1G9NAL71qAw+/5yauCfXX7Jqv\n2D7Iw++5iasHK7tmyO/j3OzCJWubNDMvgUS4MHIL97GX3uNRIJzzfAg4kXuAqp5Q1dep6m7gg+62\nhHvuPrdZLIWzVvxzgdNAUEQ6il2zlkbCQdIZ5eCJRNnnZquu1rRlTOOICJet711R18x+JrRSrcRL\nIPlr4Eci8mF3hcR/Bz7r4bxHgKvcUVZdwJuA+3IPEJENIpItwweAz+Wcu9ZNxwJOv8jj6iSoeQh4\nvbv9LcC3PJSlIrvcMeL5axB4kf0jsRqJMSbX4gJXLdRP4qWz/U+BtwFncUZJvU1V/8zDeSngnTgj\nvg4BX1HVx0TkThHJrrR4E/CkiBwGBoG73HPTOM1aD4rIAZwa0F+657wP+F0ROYLTZ+IlqFVkY7+P\nLcEe9lUQSLKjMjZZjcQYk6MV06SUHATt1hb2q+oO4CflXlxV7wfuz9v2oZzH93JhBFb+ud8DdhXY\nfhRnRNiyGAkHK6qRjCeSiMBAv61FYoy5YHC1NW2pagaIisjWZSrPijMcDjB67jynp+fKOm88kWRD\nXzed7eUuQmmMaWX93R2s6WpfzMXXCrxMy9wEPCYiP8bJ/AuAqr6m+CmtIzvpKBqL87PXDXo+z4b+\nGmMKyeYAa6UaiZdA8gd1L8UKtnMoQJuUH0hOTiYZWrumjiUzxjSrUMC3mNS1FRQNJCLyHGBQVX+Q\nt/1lOPM9VoU1XR1cPdhfdof7+GSSPdvW1qlUxphmFvL7+NHTrbPYbKkG/D8Dpgpsn3X3rRq7tzod\n7l6Xx0wupInPLiyu0GaMMbmya7dnMq2x5G6pQLJNVffnb1TVvcC2upVoBRoeCjKZTPHMmVlPx9sc\nEmNMKSG/j1RGOTMz3+ii1ESpQFLqU3BVfdXOroKWXWJzKdm1Bqyz3RhTyKC/tYYAlwokj4jIJTm1\nROTtwKP1K9LKc/VgP2u62onGvKVKubBWu80hMcZcKjtRuVUWuCo1aut3gG+IyJu5EDj24KyOeFu9\nC7aStLcJO7YEiHjscM+mPrCmLWNMIYtpUlqkRlI0kKjqSeBFIvJyYIe7+e9V9fvLUrIVZiQc5PP/\n+gxzqTTdHe0ljx2fTNLX3UF/DVZkM8a0ng193bS3Scus3b7kPBJVfQgnUeKqNhIOMp/O8MTY1GKf\nSTEnJ5MM+q1ZyxhTWHubMNDXOislWv4Ojy50uC/dvDWWsJURjTGlZYcAtwILJB5tDvgY6O/2lMDx\nZCJp/SPGmJI2tdCSuxZIPBIRhoeCREZLB5JMRjk1NWdDf40xJYUCFkhWpZFwgKMTMyRmF4oec3pm\njlRGrWnLGFPSoN/H1FyKmblUo4tSNQskZRgJO7mz9h8vXis56aaGthqJMaaU7DyzVuhwt0BShp1D\nSy+9Oz5pa7UbY5a2OLu9BZq3LJCUIdDTyRUDvSVHbo27qaGtRmKMKSWb1NVqJKvQSDhIJJYomgl4\nfDJJe5uwvs/mkRhjimultdstkJRpJBzk9PQcx+OFF6UZT8yxsd+ZtWqMMcX0dLXj93W0xMgtCyRl\nGglnl94tnMDx5KRNRjTGeNMqQ4AtkJTp2pCfrvY2okXmk9ha7cYYrwZbZO12CyRl6upoY/tmP5Fj\nRQKJzWo3xni0KeCzPpLVaiQc5MDxBKl05qLt03MppudS1rRljPEk5PcxMTV3yWdJs7FAUoGRcJDz\nC2kOn5y+aPu4rYxojCnDYMBHRmFieq7RRamKBZIKZDMB5/eT2FrtxphyLA4BbvIOdwskFdi2fg2B\nns5LZrhn/xg2WdOWMcaDVlm73QJJBUSE4XDwkhnulh7FGFOO7JdOq5GUICK3iMiTInJERN5fYP9l\nIvKgiOwXkYdFZChnX1pEIu7PfTnbPy8iT+fsG6nnPRQzEg5y+OTURZk7xxNJAj2d+DpLL8VrjDEA\n63q76GpvY3zS+kgKEpF24FPAq4HtwB0isj3vsE8Ad6vqLuBO4KM5+86r6oj785q8896bsy9Sr3so\nZSQcIKNw8PiFiYk2h8QYUw4RYaO/25q2SrgBOKKqR1V1HrgHeG3eMduBB93HDxXYv2IND1269O7J\nySSD1qxljClDyO9jLFE45VKzqGcg2QLEcp6PuttyRYHb3ce3Af0ist597hORvSLy7yJya955d7nN\nYZ8UkYZkR1zf1014Xc9FI7fGE0lCfkvWaIzxzlm73Zq2iimUtTA/Ze57gBtFZB9wI3AcyHY6bFXV\nPcB/AP5MRK50t38AuBZ4PrAOeF/BFxd5hxuI9k5MTFR3J0UMDwUXZ7gvpDNMTM8RclNDG2OMFyF3\n7fZiGcWbQT0DySgQznk+BJzIPUBVT6jq61R1N/BBd1siu8/9fRR4GNjtPh9Txxzw1zhNaJdQ1c+o\n6h5V3TMwMFDTG8saCQc5kUhyajLJxNQcqjYZ0RhTnk0BH+cX0kwmm3fJ3XoGkkeAq0TkchHpAt4E\n3Jd7gIhsEJFsGT4AfM7dvjbbZCUiG4AXA4+7zze5vwW4FThYx3soaTET8GgiZ+ivNW0ZY7xrhbkk\ndQskqpoC3gk8ABwCvqKqj4nInSKSHYV1E/CkiBwGBoG73O3XAXtFJIrTCf9Hqvq4u++LInIAOABs\nAD5Sr3tYyvWbA7S3CZHYucXlMm1WuzGmHNl5Z2NNPJeko54XV9X7gfvztn0o5/G9wL0Fzvs3YGeR\na95c42JWrKernWtD/URjCTa4KyJa05YxphyhFli73Wa2V2k4HCQ6Gmc8kaSrvY11vV2NLpIxpols\ndEd6NnM6eQskVRoZCjKVTPFvT51hMNCN03VjjDHedHe0s763ywLJajay1elwP3A8Yc1axpiKDPp9\n1rS1ml050Edvl5NbyzrajTGVCDX5SokWSKrU3ibsctOlWI3EGFOJQXdSYrOyQFID2YWuLH28MaYS\nIb+PMzPzzKXSjS5KReo6/He1GAkHAGvaMsZUJrsuyav/7J9pb6vtgJ3PvuX5bF2/pqbXzGeBpAZu\nvHojv/7Sy3nZ1fVJxWKMaW03XjPAbbu31KVG0tVR/4YnaeZEYV7t2bNH9+7d2+hiGGNMUxGRR93k\nuSVZH4kxxpiqWCAxxhhTFQskxhhjqmKBxBhjTFUskBhjjKmKBRJjjDFVsUBijDGmKhZIjDHGVGVV\nTEgUkQng2bzNG4DTDShOvbTa/UDr3ZPdz8rXavdU7f1cpqpLpuxYFYGkEBHZ62XGZrNotfuB1rsn\nu5+Vr9Xuabnux5q2jDHGVMUCiTHGmKqs5kDymUYXoMZa7X6g9e7J7mfla7V7Wpb7WbV9JMYYY2pj\nNddIjDHG1MCqCyQicouIPCkiR0Tk/Y0uTy2IyDMickBEIiLSdAuviMjnROSUiBzM2bZORL4nIj91\nf69tZBnLVeSePiwix933KSIiP9/IMpZDRMIi8pCIHBKRx0Tkt93tTfk+lbifZn6PfCLyYxGJuvf0\nB+72y0XkR+579GUR6ar5a6+mpi0RaQcOA68ARoFHgDtU9fGGFqxKIvIMsEdVm3L8u4i8DJgG7lbV\nHe62jwFnVfWP3IC/VlXf18hylqPIPX0YmFbVTzSybJUQkU3AJlX9iYj0A48CtwJvpQnfpxL380aa\n9z0SoFdVp0WkE/gX4LeB3wW+rqr3iMhfAFFV/XQtX3u11UhuAI6o6lFVnQfuAV7b4DKteqr6T8DZ\nvM2vBb7gPv4Czn/yplHknpqWqo6p6k/cx1PAIWALTfo+lbifpqWOafdpp/ujwM3Ave72urxHqy2Q\nbAFiOc9HafI/HpcC3xWRR0XkHY0uTI0MquoYOP/pgY0NLk+tvFNE9rtNX03RDJRPRLYBu4Ef0QLv\nU979QBO/RyLSLiIR4BTwPeApIK6qKfeQunzmrbZAIgW2tULb3otV9bnAq4HfcptVzMrzaeBKYAQY\nA/6kscUpn4j0AV8DfkdVJxtdnmoVuJ+mfo9UNa2qI8AQTgvMdYUOq/XrrrZAMgqEc54PAScaVJaa\nUdUT7u9TwDdw/oCa3Um3HTvbnn2qweWpmqqedP+jZ4C/pMneJ7fd/WvAF1X16+7mpn2fCt1Ps79H\nWaoaBx4GXgAERaTD3VWXz7zVFkgeAa5yRzF0AW8C7mtwmaoiIr1uZyEi0gu8EjhY+qymcB/wFvfx\nW4BvNbAsNZH9wHXdRhO9T25H7meBQ6r6pzm7mvJ9KnY/Tf4eDYhI0H3cA/wcTt/PQ8Dr3cPq8h6t\nqlFbAO5wvj8D2oHPqepdDS5SVUTkCpxaCEAH8LfNdk8i8iXgJpxMpSeB/w58E/gKsBU4BrxBVZum\n87rIPd2E02SiwDPAb2T7F1Y6EXkJ8M/AASDjbv49nH6FpnufStzPHTTve7QLpzO9HaeS8BVVvdP9\njLgHWAfsA35FVedq+tqrLZAYY4yprdXWtGWMMabGLJAYY4ypigUSY4wxVbFAYowxpioWSIwxxlTF\nAolpCSLysIi8Km/b74jI/1rivOlS+2tQrgE38+o+EXlp3r6HRWSP+3ibm531VQWu8XE3m+vHKyzD\nTSLy7ZznHxGRB0Sk2y3D3px9e0Tk4ZzzVER+KWf/t0XkpkrKYVqXBRLTKr6EM8E015vc7Y30s8AT\nqrpbVf+50AEiMgQ8ALxbVR8ocMhvAM9V1fd6ecGcWcyF9n0QeDFwa85cgo0i8uoip4wCH/Tyumb1\nskBiWsW9wC+KSDcsJuLbDPyLiPSJyIMi8hNx1m25JONzgW/tfy4ib3UfP09EfuAmxXwgb/Zz9vjL\n3NfY7/7eKiIjwMeAnxdnbYueAuUOAd8Ffl9VL8myICL3Ab3Aj0Tklwu9jnvc50XkT0XkIeCPC/0D\nici7gZ8HfklVz+fs+jjw+4XOAaJAQkReUWS/MRZITGtQ1TPAj4Fb3E1vAr6szozbJHCbm9jy5cCf\nuCkyluTmY/qfwOtV9XnA54BCmQP+HGftkV3AF4H/oaoR4ENuOUbyPryz7gb+XFW/WuS+XgOcd8//\ncqHXyTn8auDnVPXdBS71YuA/A6/OSTWe9UNgTkReXqgMwEcoHmiMsUBiWkpu81Zus5YA/5+I7Af+\nESeN9qDHa14D7AC+56bn/n2cxHf5Xgj8rfv4/wAv8Xj9fwR+VUTWeDy+1Ot8VVXTRc47gvPv8Moi\n+4sGi2yTXH4fjzFZFkhMK/km8LMi8lygJ7twEfBmYAB4npti+yTgyzs3xcX/H7L7BXjMrRGMqOpO\nVS32YZzLa+6hj+Hkq/pqqb4Nj68zU+K4kzjNWp8sVPNQ1e/j3PMLipx/F9ZXYoqwQGJahttk8zBO\n81NuJ3sAOKWqC+6H6GUFTn8W2O6OZArgdJIDPAkMiMgLwWnqEpHrC5z/b1yoDb0ZZ5lTr/4rMAl8\n1kOTW8Wvo6qHgdcBf+P23+S7C/hvRc79LrAWGPb6emb1sEBiWs2XcD7s7snZ9kVgjzvM9c3AE/kn\nqWoMJ4vtfvf4fe72eZwU3H8sIlEgAryowOv+P8Db3OazX8VZK9sTtx/nLcAmnBpKKRW/jvtajwBv\nA+4TkSvz9t0PTJQ4/S4KN+uZVc6y/xpjjKmK1UiMMcZUxQKJMcaYqlggMcYYUxULJMYYY6pigcQY\nY0xVLJAYY4ypigUSY4wxVbFAYowxpir/F19pSpGIa+FNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x26671c540f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(k_range, grid_mean_scores)\n",
    "plt.xlabel('Value of K for KNN')\n",
    "plt.ylabel('Cross-Validated Accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.98\n",
      "{'n_neighbors': 13}\n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_jobs=1, n_neighbors=13, p=2,\n",
      "           weights='uniform')\n"
     ]
    }
   ],
   "source": [
    "print(grid.best_score_)\n",
    "print(grid.best_params_)\n",
    "print(grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 搜索多个参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise',\n",
       "       estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "           weights='uniform'),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'n_neighbors': array([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,\n",
       "       18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), 'weights': ['uniform', 'distance']},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_range = np.arange(1, 31)\n",
    "weight_options = ['uniform', 'distance']\n",
    "param_grid = dict(n_neighbors=k_range, weights=weight_options)\n",
    "grid = GridSearchCV(knn, param_grid, cv=10, scoring='accuracy')\n",
    "grid.fit(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:747: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[mean: 0.96000, std: 0.05333, params: {'n_neighbors': 1, 'weights': 'uniform'},\n",
       " mean: 0.96000, std: 0.05333, params: {'n_neighbors': 1, 'weights': 'distance'},\n",
       " mean: 0.95333, std: 0.05207, params: {'n_neighbors': 2, 'weights': 'uniform'},\n",
       " mean: 0.96000, std: 0.05333, params: {'n_neighbors': 2, 'weights': 'distance'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 3, 'weights': 'uniform'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 3, 'weights': 'distance'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 4, 'weights': 'uniform'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 4, 'weights': 'distance'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 5, 'weights': 'uniform'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 5, 'weights': 'distance'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 6, 'weights': 'uniform'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 6, 'weights': 'distance'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 7, 'weights': 'uniform'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 7, 'weights': 'distance'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 8, 'weights': 'uniform'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 8, 'weights': 'distance'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 9, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 9, 'weights': 'distance'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 10, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 10, 'weights': 'distance'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 11, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 11, 'weights': 'distance'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 12, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.04422, params: {'n_neighbors': 12, 'weights': 'distance'},\n",
       " mean: 0.98000, std: 0.03055, params: {'n_neighbors': 13, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 13, 'weights': 'distance'},\n",
       " mean: 0.97333, std: 0.04422, params: {'n_neighbors': 14, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 14, 'weights': 'distance'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 15, 'weights': 'uniform'},\n",
       " mean: 0.98000, std: 0.03055, params: {'n_neighbors': 15, 'weights': 'distance'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 16, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 16, 'weights': 'distance'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 17, 'weights': 'uniform'},\n",
       " mean: 0.98000, std: 0.03055, params: {'n_neighbors': 17, 'weights': 'distance'},\n",
       " mean: 0.98000, std: 0.03055, params: {'n_neighbors': 18, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 18, 'weights': 'distance'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 19, 'weights': 'uniform'},\n",
       " mean: 0.98000, std: 0.03055, params: {'n_neighbors': 19, 'weights': 'distance'},\n",
       " mean: 0.98000, std: 0.03055, params: {'n_neighbors': 20, 'weights': 'uniform'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 20, 'weights': 'distance'},\n",
       " mean: 0.96667, std: 0.03333, params: {'n_neighbors': 21, 'weights': 'uniform'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 21, 'weights': 'distance'},\n",
       " mean: 0.96667, std: 0.03333, params: {'n_neighbors': 22, 'weights': 'uniform'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 22, 'weights': 'distance'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 23, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 23, 'weights': 'distance'},\n",
       " mean: 0.96000, std: 0.04422, params: {'n_neighbors': 24, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 24, 'weights': 'distance'},\n",
       " mean: 0.96667, std: 0.03333, params: {'n_neighbors': 25, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 25, 'weights': 'distance'},\n",
       " mean: 0.96000, std: 0.04422, params: {'n_neighbors': 26, 'weights': 'uniform'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 26, 'weights': 'distance'},\n",
       " mean: 0.96667, std: 0.04472, params: {'n_neighbors': 27, 'weights': 'uniform'},\n",
       " mean: 0.98000, std: 0.03055, params: {'n_neighbors': 27, 'weights': 'distance'},\n",
       " mean: 0.95333, std: 0.04269, params: {'n_neighbors': 28, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 28, 'weights': 'distance'},\n",
       " mean: 0.95333, std: 0.04269, params: {'n_neighbors': 29, 'weights': 'uniform'},\n",
       " mean: 0.97333, std: 0.03266, params: {'n_neighbors': 29, 'weights': 'distance'},\n",
       " mean: 0.95333, std: 0.04269, params: {'n_neighbors': 30, 'weights': 'uniform'},\n",
       " mean: 0.96667, std: 0.03333, params: {'n_neighbors': 30, 'weights': 'distance'}]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.98\n",
      "{'n_neighbors': 13, 'weights': 'uniform'}\n"
     ]
    }
   ],
   "source": [
    "print(grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用前面获得的最佳参数进行预测\n",
    "knn = KNeighborsClassifier(n_neighbors=13, weights='uniform')\n",
    "knn.fit(x, y)\n",
    "knn.predict([[3, 5, 4, 2]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 使用RandomizeSearchCV来降低计算代价"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[mean: 0.97333, std: 0.03266, params: {'weights': 'distance', 'n_neighbors': 16},\n",
       " mean: 0.96667, std: 0.03333, params: {'weights': 'uniform', 'n_neighbors': 22},\n",
       " mean: 0.98000, std: 0.03055, params: {'weights': 'uniform', 'n_neighbors': 18},\n",
       " mean: 0.96667, std: 0.04472, params: {'weights': 'uniform', 'n_neighbors': 27},\n",
       " mean: 0.95333, std: 0.04269, params: {'weights': 'uniform', 'n_neighbors': 29},\n",
       " mean: 0.97333, std: 0.03266, params: {'weights': 'distance', 'n_neighbors': 10},\n",
       " mean: 0.96667, std: 0.04472, params: {'weights': 'distance', 'n_neighbors': 22},\n",
       " mean: 0.97333, std: 0.04422, params: {'weights': 'uniform', 'n_neighbors': 14},\n",
       " mean: 0.97333, std: 0.04422, params: {'weights': 'distance', 'n_neighbors': 12},\n",
       " mean: 0.97333, std: 0.03266, params: {'weights': 'uniform', 'n_neighbors': 15}]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_dist = dict(n_neighbors=k_range, weights=weight_options)\n",
    "rand = RandomizedSearchCV(knn, param_dist, cv=10, scoring='accuracy', n_iter=10, random_state=5)\n",
    "rand.fit(x, y)\n",
    "rand.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.98\n",
      "{'weights': 'uniform', 'n_neighbors': 18}\n"
     ]
    }
   ],
   "source": [
    "print(rand.best_score_)\n",
    "print(rand.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.98,\n",
       " 0.98,\n",
       " 0.973,\n",
       " 0.973,\n",
       " 0.973,\n",
       " 0.973,\n",
       " 0.973,\n",
       " 0.973,\n",
       " 0.973,\n",
       " 0.973,\n",
       " 0.98,\n",
       " 0.98,\n",
       " 0.98,\n",
       " 0.98,\n",
       " 0.98,\n",
       " 0.973,\n",
       " 0.98,\n",
       " 0.98,\n",
       " 0.98,\n",
       " 0.973]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_scores = []\n",
    "for _ in range(20):\n",
    "    rand = RandomizedSearchCV(knn, param_dist, cv=10, scoring='accuracy', n_iter=10)\n",
    "    rand.fit(x, y)\n",
    "    best_scores.append(round(rand.best_score_, 3))\n",
    "best_scores # 虽然随机，但是大多数时候效果还是不错的。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
